{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Load the required input data\n",
    "x_train = np.load('Data/x_train.npy')\n",
    "x_test = np.load('Data/x_test.npy')\n",
    "x_train = np.squeeze(x_train)\n",
    "x_test = np.squeeze(x_test)\n",
    "\n",
    "lab_train = np.load('Data/lab_train.npy')\n",
    "lab_test = np.load('Data/lab_test.npy')\n",
    "lab_train = np.squeeze(lab_train)\n",
    "lab_test = np.squeeze(lab_test)\n",
    "\n",
    "# Load weights and biases of the autoencoder trained previously\n",
    "w = np.load('Data/ae-weights.npy')\n",
    "b = np.load('Data/ae-biases.npy')\n",
    "\n",
    "# Parameters\n",
    "learning_rate = 0.001\n",
    "training_epochs = 20\n",
    "\n",
    "# Network Parameters\n",
    "n_hidden_1 = 128\n",
    "n_hidden_2 = 50\n",
    "n_input = 256\n",
    "n_classes = 3\n",
    "\n",
    "# tf Graph input\n",
    "x = tf.placeholder(\"float\", [None, n_input])\n",
    "y = tf.placeholder(\"float\", [None, n_classes])\n",
    "\n",
    "weights = {\n",
    "    'h1': tf.constant(w[()]['encoder_h1']),\n",
    "    'h2': tf.constant(w[()]['encoder_h2']),\n",
    "    'out': tf.Variable(tf.random_normal([n_hidden_2, n_classes]))\n",
    "}\n",
    "biases = {\n",
    "    'b1': tf.constant(b[()]['encoder_b1']),\n",
    "    'b2': tf.constant(b[()]['encoder_b2']),\n",
    "    'out': tf.Variable(tf.random_normal([n_classes]))\n",
    "}\n",
    "\n",
    "# weights = {\n",
    "#     'h1': tf.Variable(tf.random_normal([n_input, n_hidden_1])),\n",
    "#     'h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2])),\n",
    "#     'out': tf.Variable(tf.random_normal([n_hidden_2, n_classes]))\n",
    "# }\n",
    "# biases = {\n",
    "#     'b1': tf.Variable(tf.random_normal([n_hidden_1])),\n",
    "#     'b2': tf.Variable(tf.random_normal([n_hidden_2])),\n",
    "#     'out': tf.Variable(tf.random_normal([n_classes]))\n",
    "# }\n",
    "\n",
    "# Create model\n",
    "def multilayer_perceptron(x, weights, biases):\n",
    "    layer_1 = tf.add(tf.matmul(x, weights['h1']), biases['b1'])\n",
    "    layer_1 = tf.nn.relu(layer_1)\n",
    "\n",
    "    layer_2 = tf.add(tf.matmul(layer_1, weights['h2']), biases['b2'])\n",
    "    layer_2 = tf.nn.relu(layer_2)\n",
    "\n",
    "    out_layer = tf.matmul(layer_2, weights['out']) + biases['out']\n",
    "    return out_layer\n",
    "\n",
    "pred = multilayer_perceptron(x, weights, biases)\n",
    "\n",
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)\n",
    "\n",
    "init = tf.initialize_all_variables()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Number of Batches: 47\n",
      "Epoch: 1 cost: 3883.88123512\n",
      "Epoch: 2 cost: 3116.37658225\n",
      "Epoch: 3 cost: 2440.31482567\n",
      "Epoch: 4 cost: 1772.4387862\n",
      "Epoch: 5 cost: 1139.20017916\n",
      "Epoch: 6 cost: 664.34558698\n",
      "Epoch: 7 cost: 363.365078784\n",
      "Epoch: 8 cost: 200.784399154\n",
      "Epoch: 9 cost: 173.637539397\n",
      "Epoch: 10 cost: 173.379376351\n",
      "Epoch: 11 cost: 168.918586082\n",
      "Epoch: 12 cost: 164.126752813\n",
      "Epoch: 13 cost: 159.179803808\n",
      "Epoch: 14 cost: 154.175374376\n",
      "Epoch: 15 cost: 149.420703563\n",
      "Epoch: 16 cost: 144.805479253\n",
      "Epoch: 17 cost: 140.078289681\n",
      "Epoch: 18 cost: 135.635014148\n",
      "Epoch: 19 cost: 131.461929727\n",
      "Epoch: 20 cost: 127.518191398\n",
      "Epoch: 21 cost: 124.010360718\n",
      "Epoch: 22 cost: 120.708243593\n",
      "Epoch: 23 cost: 117.673136123\n",
      "Epoch: 24 cost: 114.869198332\n",
      "Epoch: 25 cost: 112.399879699\n",
      "Epoch: 26 cost: 110.035990289\n",
      "Epoch: 27 cost: 107.794096358\n",
      "Epoch: 28 cost: 105.759642662\n",
      "Epoch: 29 cost: 103.89354162\n",
      "Epoch: 30 cost: 101.963668742\n",
      "Epoch: 31 cost: 100.269908986\n",
      "Epoch: 32 cost: 98.6992765386\n",
      "Epoch: 33 cost: 97.1013526105\n",
      "Epoch: 34 cost: 95.6734851269\n",
      "Epoch: 35 cost: 94.4740887906\n",
      "Epoch: 36 cost: 93.0947543205\n",
      "Epoch: 37 cost: 91.8580627441\n",
      "Epoch: 38 cost: 90.6173223942\n",
      "Epoch: 39 cost: 89.5283037551\n",
      "Epoch: 40 cost: 88.397559308\n",
      "Epoch: 41 cost: 87.3881550241\n",
      "Epoch: 42 cost: 86.3529314082\n",
      "Epoch: 43 cost: 85.3904370247\n",
      "Epoch: 44 cost: 84.3848368868\n",
      "Epoch: 45 cost: 83.443442649\n",
      "Epoch: 46 cost: 82.5630656709\n",
      "Epoch: 47 cost: 81.6162754221\n",
      "Epoch: 48 cost: 80.7325411857\n",
      "Epoch: 49 cost: 79.879425617\n",
      "Epoch: 50 cost: 79.0240153049\n",
      "Epoch: 51 cost: 78.0717305528\n",
      "Epoch: 52 cost: 77.2943677699\n",
      "Epoch: 53 cost: 76.587594499\n",
      "Epoch: 54 cost: 75.6802319872\n",
      "Epoch: 55 cost: 74.9222296451\n",
      "Epoch: 56 cost: 74.1873568677\n",
      "Epoch: 57 cost: 73.4702995787\n",
      "Epoch: 58 cost: 72.7662583209\n",
      "Epoch: 59 cost: 72.0231551718\n",
      "Epoch: 60 cost: 71.348600834\n",
      "Epoch: 61 cost: 70.599668097\n",
      "Epoch: 62 cost: 69.9513706857\n",
      "Epoch: 63 cost: 69.2672055833\n",
      "Epoch: 64 cost: 68.5824029395\n",
      "Epoch: 65 cost: 67.9177419378\n",
      "Epoch: 66 cost: 67.2199709872\n",
      "Epoch: 67 cost: 66.5656433511\n",
      "Epoch: 68 cost: 65.8853140892\n",
      "Epoch: 69 cost: 65.2388444048\n",
      "Epoch: 70 cost: 64.6299895632\n",
      "Epoch: 71 cost: 63.9840055628\n",
      "Epoch: 72 cost: 63.3264541017\n",
      "Epoch: 73 cost: 62.7408495558\n",
      "Epoch: 74 cost: 62.0875797678\n",
      "Epoch: 75 cost: 61.4399609058\n",
      "Epoch: 76 cost: 60.8012315669\n",
      "Epoch: 77 cost: 60.1708443216\n",
      "Epoch: 78 cost: 59.6081444355\n",
      "Epoch: 79 cost: 59.0815116192\n",
      "Epoch: 80 cost: 58.4661271318\n",
      "Epoch: 81 cost: 57.7984657693\n",
      "Epoch: 82 cost: 57.3096227037\n",
      "Epoch: 83 cost: 56.7176976914\n",
      "Epoch: 84 cost: 56.1825324728\n",
      "Epoch: 85 cost: 55.6518970449\n",
      "Epoch: 86 cost: 55.0984923383\n",
      "Epoch: 87 cost: 54.5658369876\n",
      "Epoch: 88 cost: 53.9586480526\n",
      "Epoch: 89 cost: 53.4239478821\n",
      "Epoch: 90 cost: 52.8584238012\n",
      "Epoch: 91 cost: 52.2830017577\n",
      "Epoch: 92 cost: 51.7424466762\n",
      "Epoch: 93 cost: 51.271991466\n",
      "Epoch: 94 cost: 50.7639443215\n",
      "Epoch: 95 cost: 50.2648088171\n",
      "Epoch: 96 cost: 49.8094235887\n",
      "Epoch: 97 cost: 49.3663647327\n",
      "Epoch: 98 cost: 48.8365941149\n",
      "Epoch: 99 cost: 48.391667082\n",
      "Epoch: 100 cost: 47.9006189894\n",
      "Epoch: 101 cost: 47.4608333872\n",
      "Epoch: 102 cost: 46.9416989266\n",
      "Epoch: 103 cost: 46.4802739874\n",
      "Epoch: 104 cost: 45.9862737351\n",
      "Epoch: 105 cost: 45.5315522742\n",
      "Epoch: 106 cost: 45.1149497641\n",
      "Epoch: 107 cost: 44.6430802142\n",
      "Epoch: 108 cost: 44.2063224569\n",
      "Epoch: 109 cost: 43.8183363143\n",
      "Epoch: 110 cost: 43.3941931116\n",
      "Epoch: 111 cost: 42.864601465\n",
      "Epoch: 112 cost: 42.4583940303\n",
      "Epoch: 113 cost: 42.0541076102\n",
      "Epoch: 114 cost: 41.7012421324\n",
      "Epoch: 115 cost: 41.2914419479\n",
      "Epoch: 116 cost: 40.9024212157\n",
      "Epoch: 117 cost: 40.5987916551\n",
      "Epoch: 118 cost: 40.3512276335\n",
      "Epoch: 119 cost: 39.9565386924\n",
      "Epoch: 120 cost: 39.5013214629\n",
      "Epoch: 121 cost: 39.1538266882\n",
      "Epoch: 122 cost: 38.7727407395\n",
      "Epoch: 123 cost: 38.4377923621\n",
      "Epoch: 124 cost: 38.1259516757\n",
      "Epoch: 125 cost: 37.9243249893\n",
      "Epoch: 126 cost: 37.774735192\n",
      "Epoch: 127 cost: 37.5648419603\n",
      "Epoch: 128 cost: 37.0089218769\n",
      "Epoch: 129 cost: 36.8776991469\n",
      "Epoch: 130 cost: 36.4738666403\n",
      "Epoch: 131 cost: 36.2699774986\n",
      "Epoch: 132 cost: 36.1039930709\n",
      "Epoch: 133 cost: 35.9508400775\n",
      "Epoch: 134 cost: 35.8498230133\n",
      "Epoch: 135 cost: 35.6573925373\n",
      "Epoch: 136 cost: 35.4393575648\n",
      "Epoch: 137 cost: 35.1989991056\n",
      "Epoch: 138 cost: 35.0916599872\n",
      "Epoch: 139 cost: 34.9270295995\n",
      "Epoch: 140 cost: 34.7669403756\n",
      "Epoch: 141 cost: 34.6592368445\n",
      "Epoch: 142 cost: 34.5689552185\n",
      "Epoch: 143 cost: 34.326350765\n",
      "Epoch: 144 cost: 34.2021941327\n",
      "Epoch: 145 cost: 34.0760552807\n",
      "Epoch: 146 cost: 33.9489948572\n",
      "Epoch: 147 cost: 33.7627758447\n",
      "Epoch: 148 cost: 33.6035763735\n",
      "Epoch: 149 cost: 33.3938468619\n",
      "Epoch: 150 cost: 33.1676761876\n",
      "Epoch: 151 cost: 33.0014064819\n",
      "Epoch: 152 cost: 32.8468915275\n",
      "Epoch: 153 cost: 32.6914284635\n",
      "Epoch: 154 cost: 32.4593422388\n",
      "Epoch: 155 cost: 32.1831758048\n",
      "Epoch: 156 cost: 32.0420887622\n",
      "Epoch: 157 cost: 31.9216170387\n",
      "Epoch: 158 cost: 31.7983623743\n",
      "Epoch: 159 cost: 31.6701529736\n",
      "Epoch: 160 cost: 31.6198165341\n",
      "Epoch: 161 cost: 31.4984160256\n",
      "Epoch: 162 cost: 31.3845822431\n",
      "Epoch: 163 cost: 31.2748133629\n",
      "Epoch: 164 cost: 31.1704422337\n",
      "Epoch: 165 cost: 31.0911934858\n",
      "Epoch: 166 cost: 30.9810373808\n",
      "Epoch: 167 cost: 30.8497611893\n",
      "Epoch: 168 cost: 30.7388255748\n",
      "Epoch: 169 cost: 30.6257143401\n",
      "Epoch: 170 cost: 30.5248860222\n",
      "Epoch: 171 cost: 30.3800689362\n",
      "Epoch: 172 cost: 30.2653898102\n",
      "Epoch: 173 cost: 30.1322234818\n",
      "Epoch: 174 cost: 30.0107974271\n",
      "Epoch: 175 cost: 29.9166798363\n",
      "Epoch: 176 cost: 29.8343032522\n",
      "Epoch: 177 cost: 29.7382823939\n",
      "Epoch: 178 cost: 29.6250854832\n",
      "Epoch: 179 cost: 29.5150892278\n",
      "Epoch: 180 cost: 29.3976007928\n",
      "Epoch: 181 cost: 29.2663223769\n",
      "Epoch: 182 cost: 29.1775284295\n",
      "Epoch: 183 cost: 29.1058081541\n",
      "Epoch: 184 cost: 29.0100177856\n",
      "Epoch: 185 cost: 28.9429967682\n",
      "Epoch: 186 cost: 28.8998867857\n",
      "Epoch: 187 cost: 28.8358554307\n",
      "Epoch: 188 cost: 28.7168413046\n",
      "Epoch: 189 cost: 28.6246321024\n",
      "Epoch: 190 cost: 28.5395668111\n",
      "Epoch: 191 cost: 28.4620897795\n",
      "Epoch: 192 cost: 28.4045066884\n",
      "Epoch: 193 cost: 28.3584551304\n",
      "Epoch: 194 cost: 28.2867614589\n",
      "Epoch: 195 cost: 28.2120211556\n",
      "Epoch: 196 cost: 28.1329385042\n",
      "Epoch: 197 cost: 28.0448685174\n",
      "Epoch: 198 cost: 27.963850034\n",
      "Epoch: 199 cost: 27.8950000038\n",
      "Epoch: 200 cost: 27.8323725386\n",
      "Epoch: 201 cost: 27.7756290613\n",
      "Epoch: 202 cost: 27.6897750636\n",
      "Epoch: 203 cost: 27.5992693267\n",
      "Epoch: 204 cost: 27.521789112\n",
      "Epoch: 205 cost: 27.4459829051\n",
      "Epoch: 206 cost: 27.3695237789\n",
      "Epoch: 207 cost: 27.2958463735\n",
      "Epoch: 208 cost: 27.232908221\n",
      "Epoch: 209 cost: 27.1427547729\n",
      "Epoch: 210 cost: 27.0572740677\n",
      "Epoch: 211 cost: 26.9938840333\n",
      "Epoch: 212 cost: 26.93661532\n",
      "Epoch: 213 cost: 26.8768754665\n",
      "Epoch: 214 cost: 26.814625111\n",
      "Epoch: 215 cost: 26.7543471164\n",
      "Epoch: 216 cost: 26.695291613\n",
      "Epoch: 217 cost: 26.6230270837\n",
      "Epoch: 218 cost: 26.4855392588\n",
      "Epoch: 219 cost: 26.3882895759\n",
      "Epoch: 220 cost: 26.3374353622\n",
      "Epoch: 221 cost: 26.2700895654\n",
      "Epoch: 222 cost: 26.217158523\n",
      "Epoch: 223 cost: 26.1773904511\n",
      "Epoch: 224 cost: 26.1383561758\n",
      "Epoch: 225 cost: 26.0983510246\n",
      "Epoch: 226 cost: 26.0568987917\n",
      "Epoch: 227 cost: 26.0135132734\n",
      "Epoch: 228 cost: 25.9675755831\n",
      "Epoch: 229 cost: 25.9094277012\n",
      "Epoch: 230 cost: 25.8412355337\n",
      "Epoch: 231 cost: 25.7981170289\n",
      "Epoch: 232 cost: 25.7554956801\n",
      "Epoch: 233 cost: 25.7136433023\n",
      "Epoch: 234 cost: 25.6791882566\n",
      "Epoch: 235 cost: 25.6450462392\n",
      "Epoch: 236 cost: 25.6095576159\n",
      "Epoch: 237 cost: 25.5709521491\n",
      "Epoch: 238 cost: 25.5193447737\n",
      "Epoch: 239 cost: 25.4677871948\n",
      "Epoch: 240 cost: 25.4196865432\n",
      "Epoch: 241 cost: 25.3763262211\n",
      "Epoch: 242 cost: 25.3368325259\n",
      "Epoch: 243 cost: 25.2795125373\n",
      "Epoch: 244 cost: 25.2013400337\n",
      "Epoch: 245 cost: 25.1504784949\n",
      "Epoch: 246 cost: 25.1093401655\n",
      "Epoch: 247 cost: 25.0740780906\n",
      "Epoch: 248 cost: 25.041572931\n",
      "Epoch: 249 cost: 25.0088520684\n",
      "Epoch: 250 cost: 24.9765852132\n",
      "Epoch: 251 cost: 24.9455007239\n",
      "Epoch: 252 cost: 24.9151672835\n",
      "Epoch: 253 cost: 24.8848924916\n",
      "Epoch: 254 cost: 24.8528851778\n",
      "Epoch: 255 cost: 24.8177799722\n",
      "Epoch: 256 cost: 24.7812923878\n",
      "Epoch: 257 cost: 24.7461939923\n",
      "Epoch: 258 cost: 24.7115150538\n",
      "Epoch: 259 cost: 24.6757609743\n",
      "Epoch: 260 cost: 24.6410805266\n",
      "Epoch: 261 cost: 24.6115157782\n",
      "Epoch: 262 cost: 24.5865216433\n",
      "Epoch: 263 cost: 24.5608905985\n",
      "Epoch: 264 cost: 24.530495357\n",
      "Epoch: 265 cost: 24.4954122482\n",
      "Epoch: 266 cost: 24.4647246244\n",
      "Epoch: 267 cost: 24.4505037399\n",
      "Epoch: 268 cost: 24.4221500868\n",
      "Epoch: 269 cost: 24.3949398284\n",
      "Epoch: 270 cost: 24.3724708303\n",
      "Epoch: 271 cost: 24.345996494\n",
      "Epoch: 272 cost: 24.3215909182\n",
      "Epoch: 273 cost: 24.292513264\n",
      "Epoch: 274 cost: 24.2588721742\n",
      "Epoch: 275 cost: 24.2269298574\n",
      "Epoch: 276 cost: 24.1973710466\n",
      "Epoch: 277 cost: 24.1699540438\n",
      "Epoch: 278 cost: 24.143826822\n",
      "Epoch: 279 cost: 24.1184983761\n",
      "Epoch: 280 cost: 24.0935784781\n",
      "Epoch: 281 cost: 24.0686921262\n",
      "Epoch: 282 cost: 24.0436540665\n",
      "Epoch: 283 cost: 24.0179812071\n",
      "Epoch: 284 cost: 23.9908184625\n",
      "Epoch: 285 cost: 23.9604309493\n",
      "Epoch: 286 cost: 23.9256812207\n",
      "Epoch: 287 cost: 23.891516546\n",
      "Epoch: 288 cost: 23.8619762558\n",
      "Epoch: 289 cost: 23.8353445606\n",
      "Epoch: 290 cost: 23.8089040492\n",
      "Epoch: 291 cost: 23.7813599389\n",
      "Epoch: 292 cost: 23.7514158183\n",
      "Epoch: 293 cost: 23.7185931104\n",
      "Epoch: 294 cost: 23.6839689448\n",
      "Epoch: 295 cost: 23.6520380847\n",
      "Epoch: 296 cost: 23.6273488719\n",
      "Epoch: 297 cost: 23.6088624939\n",
      "Epoch: 298 cost: 23.5925025813\n",
      "Epoch: 299 cost: 23.5756308551\n",
      "Epoch: 300 cost: 23.5574788109\n",
      "Epoch: 301 cost: 23.5384493762\n",
      "Epoch: 302 cost: 23.5189006988\n",
      "Epoch: 303 cost: 23.4989381902\n",
      "Epoch: 304 cost: 23.4789444487\n",
      "Epoch: 305 cost: 23.4588794683\n",
      "Epoch: 306 cost: 23.4387975551\n",
      "Epoch: 307 cost: 23.4185411372\n",
      "Epoch: 308 cost: 23.3978301312\n",
      "Epoch: 309 cost: 23.3762460998\n",
      "Epoch: 310 cost: 23.3538959001\n",
      "Epoch: 311 cost: 23.3314638493\n",
      "Epoch: 312 cost: 23.3097122527\n",
      "Epoch: 313 cost: 23.290375603\n",
      "Epoch: 314 cost: 23.2713529886\n",
      "Epoch: 315 cost: 23.2458499492\n",
      "Epoch: 316 cost: 23.2111571469\n",
      "Epoch: 317 cost: 23.1667954871\n",
      "Epoch: 318 cost: 23.116177044\n",
      "Epoch: 319 cost: 23.0607243447\n",
      "Epoch: 320 cost: 22.9944568492\n",
      "Epoch: 321 cost: 22.9083848456\n",
      "Epoch: 322 cost: 22.8211677252\n",
      "Epoch: 323 cost: 22.773452807\n",
      "Epoch: 324 cost: 22.7586917852\n",
      "Epoch: 325 cost: 22.7505807547\n",
      "Epoch: 326 cost: 22.7420645749\n",
      "Epoch: 327 cost: 22.7331496401\n",
      "Epoch: 328 cost: 22.7233696161\n",
      "Epoch: 329 cost: 22.7125903495\n",
      "Epoch: 330 cost: 22.7013018004\n",
      "Epoch: 331 cost: 22.6903067218\n",
      "Epoch: 332 cost: 22.6802599455\n",
      "Epoch: 333 cost: 22.6708272315\n",
      "Epoch: 334 cost: 22.6613257971\n",
      "Epoch: 335 cost: 22.6509494984\n",
      "Epoch: 336 cost: 22.6389561232\n",
      "Epoch: 337 cost: 22.6238001991\n",
      "Epoch: 338 cost: 22.6036137774\n",
      "Epoch: 339 cost: 22.5798665158\n",
      "Epoch: 340 cost: 22.5589371286\n",
      "Epoch: 341 cost: 22.5427166147\n",
      "Epoch: 342 cost: 22.5287859947\n",
      "Epoch: 343 cost: 22.515954847\n",
      "Epoch: 344 cost: 22.5037065389\n",
      "Epoch: 345 cost: 22.4914511594\n",
      "Epoch: 346 cost: 22.4787407378\n",
      "Epoch: 347 cost: 22.4652680615\n",
      "Epoch: 348 cost: 22.4508235886\n",
      "Epoch: 349 cost: 22.4353917279\n",
      "Epoch: 350 cost: 22.4195228561\n",
      "Epoch: 351 cost: 22.4036237128\n",
      "Epoch: 352 cost: 22.3877648851\n",
      "Epoch: 353 cost: 22.3718995515\n",
      "Epoch: 354 cost: 22.3559972606\n",
      "Epoch: 355 cost: 22.3396829494\n",
      "Epoch: 356 cost: 22.3228426741\n",
      "Epoch: 357 cost: 22.3055328537\n",
      "Epoch: 358 cost: 22.287844384\n",
      "Epoch: 359 cost: 22.2697104723\n",
      "Epoch: 360 cost: 22.2518983699\n",
      "Epoch: 361 cost: 22.2351076704\n",
      "Epoch: 362 cost: 22.2172007916\n",
      "Epoch: 363 cost: 22.1976301442\n",
      "Epoch: 364 cost: 22.1780803026\n",
      "Epoch: 365 cost: 22.1592890278\n",
      "Epoch: 366 cost: 22.1405168949\n",
      "Epoch: 367 cost: 22.1193232384\n",
      "Epoch: 368 cost: 22.0926205884\n",
      "Epoch: 369 cost: 22.0614726062\n",
      "Epoch: 370 cost: 22.0350434121\n",
      "Epoch: 371 cost: 22.0168085225\n",
      "Epoch: 372 cost: 22.0030161954\n",
      "Epoch: 373 cost: 21.9914203527\n",
      "Epoch: 374 cost: 21.9811890024\n",
      "Epoch: 375 cost: 21.9721299537\n",
      "Epoch: 376 cost: 21.963635366\n",
      "Epoch: 377 cost: 21.9553294588\n",
      "Epoch: 378 cost: 21.9468858014\n",
      "Epoch: 379 cost: 21.9380721209\n",
      "Epoch: 380 cost: 21.9290831901\n",
      "Epoch: 381 cost: 21.9197526802\n",
      "Epoch: 382 cost: 21.9100134817\n",
      "Epoch: 383 cost: 21.8994549196\n",
      "Epoch: 384 cost: 21.8877887066\n",
      "Epoch: 385 cost: 21.8751793783\n",
      "Epoch: 386 cost: 21.8617273645\n",
      "Epoch: 387 cost: 21.8479959052\n",
      "Epoch: 388 cost: 21.8343503247\n",
      "Epoch: 389 cost: 21.820571907\n",
      "Epoch: 390 cost: 21.8066693544\n",
      "Epoch: 391 cost: 21.7923752792\n",
      "Epoch: 392 cost: 21.7772156863\n",
      "Epoch: 393 cost: 21.7609754294\n",
      "Epoch: 394 cost: 21.7435347679\n",
      "Epoch: 395 cost: 21.7247749567\n",
      "Epoch: 396 cost: 21.7048697636\n",
      "Epoch: 397 cost: 21.6839755609\n",
      "Epoch: 398 cost: 21.6625396178\n",
      "Epoch: 399 cost: 21.6410419218\n",
      "Epoch: 400 cost: 21.6201591314\n",
      "Epoch: 401 cost: 21.6006625143\n",
      "Epoch: 402 cost: 21.5824781009\n",
      "Epoch: 403 cost: 21.5655383277\n",
      "Epoch: 404 cost: 21.5494222844\n",
      "Epoch: 405 cost: 21.5336170869\n",
      "Epoch: 406 cost: 21.5178600347\n",
      "Epoch: 407 cost: 21.5019708745\n",
      "Epoch: 408 cost: 21.4862883421\n",
      "Epoch: 409 cost: 21.4710887113\n",
      "Epoch: 410 cost: 21.4568893402\n",
      "Epoch: 411 cost: 21.4440719815\n",
      "Epoch: 412 cost: 21.4327488072\n",
      "Epoch: 413 cost: 21.4228640085\n",
      "Epoch: 414 cost: 21.4142736095\n",
      "Epoch: 415 cost: 21.4063769756\n",
      "Epoch: 416 cost: 21.3990056248\n",
      "Epoch: 417 cost: 21.3919080509\n",
      "Epoch: 418 cost: 21.3852633289\n",
      "Epoch: 419 cost: 21.3791634047\n",
      "Epoch: 420 cost: 21.3742205856\n",
      "Epoch: 421 cost: 21.3713744663\n",
      "Epoch: 422 cost: 21.3722559023\n",
      "Epoch: 423 cost: 21.3776317632\n",
      "Epoch: 424 cost: 21.3876417723\n",
      "Epoch: 425 cost: 21.4006722861\n",
      "Epoch: 426 cost: 21.4112412042\n",
      "Epoch: 427 cost: 21.4136366819\n",
      "Epoch: 428 cost: 21.4050771417\n",
      "Epoch: 429 cost: 21.3857136622\n",
      "Epoch: 430 cost: 21.3559496897\n",
      "Epoch: 431 cost: 21.3187546312\n",
      "Epoch: 432 cost: 21.2805360835\n",
      "Epoch: 433 cost: 21.2482546479\n",
      "Epoch: 434 cost: 21.2273750571\n",
      "Epoch: 435 cost: 21.2191124863\n",
      "Epoch: 436 cost: 21.2219475558\n",
      "Epoch: 437 cost: 21.2376146545\n",
      "Epoch: 438 cost: 21.270538292\n",
      "Epoch: 439 cost: 21.3120311511\n",
      "Epoch: 440 cost: 21.3393107336\n",
      "Epoch: 441 cost: 21.3421712657\n",
      "Epoch: 442 cost: 21.3196730981\n",
      "Epoch: 443 cost: 21.2977324179\n",
      "Epoch: 444 cost: 21.2780661596\n",
      "Epoch: 445 cost: 21.2466465298\n",
      "Epoch: 446 cost: 21.1947079085\n",
      "Epoch: 447 cost: 21.129259041\n",
      "Epoch: 448 cost: 21.0860901779\n",
      "Epoch: 449 cost: 21.073975052\n",
      "Epoch: 450 cost: 21.0747877131\n",
      "Epoch: 451 cost: 21.0886753747\n",
      "Epoch: 452 cost: 21.1246768084\n",
      "Epoch: 453 cost: 21.1713427622\n",
      "Epoch: 454 cost: 21.190507327\n",
      "Epoch: 455 cost: 21.1902978877\n",
      "Epoch: 456 cost: 21.1675441569\n",
      "Epoch: 457 cost: 21.1259326136\n",
      "Epoch: 458 cost: 21.0723056768\n",
      "Epoch: 459 cost: 21.0046628815\n",
      "Epoch: 460 cost: 20.9607134601\n",
      "Epoch: 461 cost: 20.9510238805\n",
      "Epoch: 462 cost: 20.9551828554\n",
      "Epoch: 463 cost: 20.9748629547\n",
      "Epoch: 464 cost: 21.0171605856\n",
      "Epoch: 465 cost: 21.0574508119\n",
      "Epoch: 466 cost: 21.0711118655\n",
      "Epoch: 467 cost: 21.0655676469\n",
      "Epoch: 468 cost: 21.037017249\n",
      "Epoch: 469 cost: 20.9977763777\n",
      "Epoch: 470 cost: 20.9462959576\n",
      "Epoch: 471 cost: 20.8876404077\n",
      "Epoch: 472 cost: 20.8582668266\n",
      "Epoch: 473 cost: 20.8569953581\n",
      "Epoch: 474 cost: 20.8749671807\n",
      "Epoch: 475 cost: 20.915797283\n",
      "Epoch: 476 cost: 20.9599586586\n",
      "Epoch: 477 cost: 20.9870148288\n",
      "Epoch: 478 cost: 20.9919265458\n",
      "Epoch: 479 cost: 20.9623582718\n",
      "Epoch: 480 cost: 20.9243956246\n",
      "Epoch: 481 cost: 20.8856940079\n",
      "Epoch: 482 cost: 20.8304186577\n",
      "Epoch: 483 cost: 20.7740124401\n",
      "Epoch: 484 cost: 20.7480822992\n",
      "Epoch: 485 cost: 20.7501616427\n",
      "Epoch: 486 cost: 20.7765045458\n",
      "Epoch: 487 cost: 20.8211456246\n",
      "Epoch: 488 cost: 20.8492823116\n",
      "Epoch: 489 cost: 20.8564160796\n",
      "Epoch: 490 cost: 20.8414039244\n",
      "Epoch: 491 cost: 20.8092614514\n",
      "Epoch: 492 cost: 20.7748237442\n",
      "Epoch: 493 cost: 20.7357882997\n",
      "Epoch: 494 cost: 20.6945603135\n",
      "Epoch: 495 cost: 20.6647817442\n",
      "Epoch: 496 cost: 20.6537321717\n",
      "Epoch: 497 cost: 20.6570068687\n",
      "Epoch: 498 cost: 20.66616923\n",
      "Epoch: 499 cost: 20.6738635783\n",
      "Epoch: 500 cost: 20.6750845491\n",
      "Epoch: 501 cost: 20.6693289914\n",
      "Epoch: 502 cost: 20.6591314851\n",
      "Epoch: 503 cost: 20.6472675052\n",
      "Epoch: 504 cost: 20.6352698714\n",
      "Epoch: 505 cost: 20.6245339741\n",
      "Epoch: 506 cost: 20.615474985\n",
      "Epoch: 507 cost: 20.6081675367\n",
      "Epoch: 508 cost: 20.6023624641\n",
      "Epoch: 509 cost: 20.5971140722\n",
      "Epoch: 510 cost: 20.5909728573\n",
      "Epoch: 511 cost: 20.5831372776\n",
      "Epoch: 512 cost: 20.5736644547\n",
      "Epoch: 513 cost: 20.5625027634\n",
      "Epoch: 514 cost: 20.5502407462\n",
      "Epoch: 515 cost: 20.5374016977\n",
      "Epoch: 516 cost: 20.5239826276\n",
      "Epoch: 517 cost: 20.5103459028\n",
      "Epoch: 518 cost: 20.4965472323\n",
      "Epoch: 519 cost: 20.4828310926\n",
      "Epoch: 520 cost: 20.4690562626\n",
      "Epoch: 521 cost: 20.4550102442\n",
      "Epoch: 522 cost: 20.4408637057\n",
      "Epoch: 523 cost: 20.4269525193\n",
      "Epoch: 524 cost: 20.4137920864\n",
      "Epoch: 525 cost: 20.4013742728\n",
      "Epoch: 526 cost: 20.3899960645\n",
      "Epoch: 527 cost: 20.3803605082\n",
      "Epoch: 528 cost: 20.3720789907\n",
      "Epoch: 529 cost: 20.3647232449\n",
      "Epoch: 530 cost: 20.3576305762\n",
      "Epoch: 531 cost: 20.3495042426\n",
      "Epoch: 532 cost: 20.3395466411\n",
      "Epoch: 533 cost: 20.3263070824\n",
      "Epoch: 534 cost: 20.3081969327\n",
      "Epoch: 535 cost: 20.2828910363\n",
      "Epoch: 536 cost: 20.2484340249\n",
      "Epoch: 537 cost: 20.2067486504\n",
      "Epoch: 538 cost: 20.1703190334\n",
      "Epoch: 539 cost: 20.1502706193\n",
      "Epoch: 540 cost: 20.1445368516\n",
      "Epoch: 541 cost: 20.1494694936\n",
      "Epoch: 542 cost: 20.1650971542\n",
      "Epoch: 543 cost: 20.1945058551\n",
      "Epoch: 544 cost: 20.2366852887\n",
      "Epoch: 545 cost: 20.2716233768\n",
      "Epoch: 546 cost: 20.2709896653\n",
      "Epoch: 547 cost: 20.2382169531\n",
      "Epoch: 548 cost: 20.2083584856\n",
      "Epoch: 549 cost: 20.1882035263\n",
      "Epoch: 550 cost: 20.1721959938\n",
      "Epoch: 551 cost: 20.1513900719\n",
      "Epoch: 552 cost: 20.1163498265\n",
      "Epoch: 553 cost: 20.0690811276\n",
      "Epoch: 554 cost: 20.02635542\n",
      "Epoch: 555 cost: 20.0009475452\n",
      "Epoch: 556 cost: 19.9915678336\n",
      "Epoch: 557 cost: 19.9911105405\n",
      "Epoch: 558 cost: 19.9949093677\n",
      "Epoch: 559 cost: 20.0060247107\n",
      "Epoch: 560 cost: 20.0320834568\n",
      "Epoch: 561 cost: 20.0791688929\n",
      "Epoch: 562 cost: 20.1333678504\n",
      "Epoch: 563 cost: 20.1562873315\n",
      "Epoch: 564 cost: 20.0992593245\n",
      "Epoch: 565 cost: 20.0279420627\n",
      "Epoch: 566 cost: 19.9755367561\n",
      "Epoch: 567 cost: 19.9460807899\n",
      "Epoch: 568 cost: 19.9260704695\n",
      "Epoch: 569 cost: 19.8887418065\n",
      "Epoch: 570 cost: 19.8453102353\n",
      "Epoch: 571 cost: 19.8003063392\n",
      "Epoch: 572 cost: 19.7476185307\n",
      "Epoch: 573 cost: 19.7136698961\n",
      "Epoch: 574 cost: 19.7066340751\n",
      "Epoch: 575 cost: 19.7012172062\n",
      "Epoch: 576 cost: 19.7416224061\n",
      "Epoch: 577 cost: 19.7811990609\n",
      "Epoch: 578 cost: 19.8243395889\n",
      "Epoch: 579 cost: 19.8655571113\n",
      "Epoch: 580 cost: 19.8784303551\n",
      "Epoch: 581 cost: 19.9166324367\n",
      "Epoch: 582 cost: 19.9411687255\n",
      "Epoch: 583 cost: 19.9680687405\n",
      "Epoch: 584 cost: 19.9821964857\n",
      "Epoch: 585 cost: 19.9866694859\n",
      "Epoch: 586 cost: 19.937339434\n",
      "Epoch: 587 cost: 19.8538674763\n",
      "Epoch: 588 cost: 19.778324202\n",
      "Epoch: 589 cost: 19.7536479323\n",
      "Epoch: 590 cost: 19.7506158542\n",
      "Epoch: 591 cost: 19.7550651723\n",
      "Epoch: 592 cost: 19.6969254309\n",
      "Epoch: 593 cost: 19.7311512359\n",
      "Epoch: 594 cost: 19.7202702089\n",
      "Epoch: 595 cost: 19.7369352604\n",
      "Epoch: 596 cost: 19.7508809047\n",
      "Epoch: 597 cost: 19.7671265247\n",
      "Epoch: 598 cost: 19.7707883804\n",
      "Epoch: 599 cost: 19.7588902395\n",
      "Epoch: 600 cost: 19.7335335245\n",
      "Epoch: 601 cost: 19.7066536165\n",
      "Epoch: 602 cost: 19.6829884547\n",
      "Epoch: 603 cost: 19.6627694647\n",
      "Epoch: 604 cost: 19.6460953165\n",
      "Epoch: 605 cost: 19.6296888869\n",
      "Epoch: 606 cost: 19.614221129\n",
      "Epoch: 607 cost: 19.6043169689\n",
      "Epoch: 608 cost: 19.5921075205\n",
      "Epoch: 609 cost: 19.5815834682\n",
      "Epoch: 610 cost: 19.5709740215\n",
      "Epoch: 611 cost: 19.5580681905\n",
      "Epoch: 612 cost: 19.5469047985\n",
      "Epoch: 613 cost: 19.5280280519\n",
      "Epoch: 614 cost: 19.5309990987\n",
      "Epoch: 615 cost: 19.4937809541\n",
      "Epoch: 616 cost: 19.5382538045\n",
      "Epoch: 617 cost: 19.495837521\n",
      "Epoch: 618 cost: 19.5342050284\n",
      "Epoch: 619 cost: 19.516797398\n",
      "Epoch: 620 cost: 19.5352021316\n",
      "Epoch: 621 cost: 19.5410850086\n",
      "Epoch: 622 cost: 19.5433425092\n",
      "Epoch: 623 cost: 19.5387709014\n",
      "Epoch: 624 cost: 19.5290822437\n",
      "Epoch: 625 cost: 19.5172507547\n",
      "Epoch: 626 cost: 19.5039088688\n",
      "Epoch: 627 cost: 19.4908314626\n",
      "Epoch: 628 cost: 19.4743079023\n",
      "Epoch: 629 cost: 19.4638267388\n",
      "Epoch: 630 cost: 19.4403651661\n",
      "Epoch: 631 cost: 19.4545447281\n",
      "Epoch: 632 cost: 19.3809383448\n",
      "Epoch: 633 cost: 19.4369902738\n",
      "Epoch: 634 cost: 19.3762554044\n",
      "Epoch: 635 cost: 19.4278142414\n",
      "Epoch: 636 cost: 19.3758959542\n",
      "Epoch: 637 cost: 19.4170110758\n",
      "Epoch: 638 cost: 19.3731088321\n",
      "Epoch: 639 cost: 19.4047300727\n",
      "Epoch: 640 cost: 19.3582550503\n",
      "Epoch: 641 cost: 19.3885460572\n",
      "Epoch: 642 cost: 19.3362589364\n",
      "Epoch: 643 cost: 19.3699295711\n",
      "Epoch: 644 cost: 19.3109641925\n",
      "Epoch: 645 cost: 19.350018235\n",
      "Epoch: 646 cost: 19.2883275468\n",
      "Epoch: 647 cost: 19.3304476776\n",
      "Epoch: 648 cost: 19.2689029156\n",
      "Epoch: 649 cost: 19.3117412963\n",
      "Epoch: 650 cost: 19.2513424044\n",
      "Epoch: 651 cost: 19.2941248582\n",
      "Epoch: 652 cost: 19.2341357546\n",
      "Epoch: 653 cost: 19.2775163359\n",
      "Epoch: 654 cost: 19.2171852956\n",
      "Epoch: 655 cost: 19.2619258837\n",
      "Epoch: 656 cost: 19.2004976793\n",
      "Epoch: 657 cost: 19.2466315247\n",
      "Epoch: 658 cost: 19.1836786549\n",
      "Epoch: 659 cost: 19.2301960156\n",
      "Epoch: 660 cost: 19.1655895761\n",
      "Epoch: 661 cost: 19.2117634314\n",
      "Epoch: 662 cost: 19.1460207866\n",
      "Epoch: 663 cost: 19.1940994795\n",
      "Epoch: 664 cost: 19.1282015798\n",
      "Epoch: 665 cost: 19.1808128788\n",
      "Epoch: 666 cost: 19.1159652129\n",
      "Epoch: 667 cost: 19.1721212395\n",
      "Epoch: 668 cost: 19.1093991703\n",
      "Epoch: 669 cost: 19.1658524579\n",
      "Epoch: 670 cost: 19.1062062548\n",
      "Epoch: 671 cost: 19.1597384671\n",
      "Epoch: 672 cost: 19.1016218371\n",
      "Epoch: 673 cost: 19.1515346804\n",
      "Epoch: 674 cost: 19.0896577201\n",
      "Epoch: 675 cost: 19.1401231137\n",
      "Epoch: 676 cost: 19.0693690244\n",
      "Epoch: 677 cost: 19.1260988395\n",
      "Epoch: 678 cost: 19.0464226642\n",
      "Epoch: 679 cost: 19.1103532124\n",
      "Epoch: 680 cost: 19.0243541256\n",
      "Epoch: 681 cost: 19.0929674176\n",
      "Epoch: 682 cost: 19.0032580685\n",
      "Epoch: 683 cost: 19.0748532896\n",
      "Epoch: 684 cost: 18.9847441709\n",
      "Epoch: 685 cost: 19.0575502388\n",
      "Epoch: 686 cost: 18.9682319874\n",
      "Epoch: 687 cost: 19.0403065149\n",
      "Epoch: 688 cost: 18.952008761\n",
      "Epoch: 689 cost: 19.0213312228\n",
      "Epoch: 690 cost: 18.9347432408\n",
      "Epoch: 691 cost: 18.9994031939\n",
      "Epoch: 692 cost: 18.9165042017\n",
      "Epoch: 693 cost: 18.975874301\n",
      "Epoch: 694 cost: 18.8983420235\n",
      "Epoch: 695 cost: 18.9552647776\n",
      "Epoch: 696 cost: 18.8818809111\n",
      "Epoch: 697 cost: 18.9371324118\n",
      "Epoch: 698 cost: 18.8659371886\n",
      "Epoch: 699 cost: 18.9158522056\n",
      "Epoch: 700 cost: 18.8455954828\n",
      "Epoch: 701 cost: 18.8887256574\n",
      "Epoch: 702 cost: 18.8251864327\n",
      "Epoch: 703 cost: 18.866272575\n",
      "Epoch: 704 cost: 18.8363044097\n",
      "Epoch: 705 cost: 18.8753122979\n",
      "Epoch: 706 cost: 18.8520889498\n",
      "Epoch: 707 cost: 18.8571688761\n",
      "Epoch: 708 cost: 18.8411548201\n",
      "Epoch: 709 cost: 18.844181544\n",
      "Epoch: 710 cost: 18.8415437658\n",
      "Epoch: 711 cost: 18.8297112394\n",
      "Epoch: 712 cost: 18.8270949886\n",
      "Epoch: 713 cost: 18.8283618194\n",
      "Epoch: 714 cost: 18.8176182876\n",
      "Epoch: 715 cost: 18.8080998773\n",
      "Epoch: 716 cost: 18.8126059454\n",
      "Epoch: 717 cost: 18.8087518418\n",
      "Epoch: 718 cost: 18.7898952035\n",
      "Epoch: 719 cost: 18.7908590106\n",
      "Epoch: 720 cost: 18.8006409254\n",
      "Epoch: 721 cost: 18.7786304824\n",
      "Epoch: 722 cost: 18.7661635571\n",
      "Epoch: 723 cost: 18.7826927812\n",
      "Epoch: 724 cost: 18.7728614642\n",
      "Epoch: 725 cost: 18.7481066559\n",
      "Epoch: 726 cost: 18.7595045275\n",
      "Epoch: 727 cost: 18.7575868076\n",
      "Epoch: 728 cost: 18.7394153729\n",
      "Epoch: 729 cost: 18.7412090111\n",
      "Epoch: 730 cost: 18.7339076653\n",
      "Epoch: 731 cost: 18.7293163614\n",
      "Epoch: 732 cost: 18.7215378246\n",
      "Epoch: 733 cost: 18.717590119\n",
      "Epoch: 734 cost: 18.7080595265\n",
      "Epoch: 735 cost: 18.7075802418\n",
      "Epoch: 736 cost: 18.6892524222\n",
      "Epoch: 737 cost: 18.706187129\n",
      "Epoch: 738 cost: 18.6550434422\n",
      "Epoch: 739 cost: 18.7164148612\n",
      "Epoch: 740 cost: 18.6366513663\n",
      "Epoch: 741 cost: 18.6937570306\n",
      "Epoch: 742 cost: 18.6238206955\n",
      "Epoch: 743 cost: 18.673100332\n",
      "Epoch: 744 cost: 18.6100609784\n",
      "Epoch: 745 cost: 18.6544301091\n",
      "Epoch: 746 cost: 18.5965594827\n",
      "Epoch: 747 cost: 18.6360823322\n",
      "Epoch: 748 cost: 18.5833946073\n",
      "Epoch: 749 cost: 18.6197147547\n",
      "Epoch: 750 cost: 18.5709979319\n",
      "Epoch: 751 cost: 18.6064331696\n",
      "Epoch: 752 cost: 18.5594682947\n",
      "Epoch: 753 cost: 18.5940311906\n",
      "Epoch: 754 cost: 18.5480882323\n",
      "Epoch: 755 cost: 18.5824573826\n",
      "Epoch: 756 cost: 18.5368034599\n",
      "Epoch: 757 cost: 18.5710823942\n",
      "Epoch: 758 cost: 18.5254995557\n",
      "Epoch: 759 cost: 18.5590841098\n",
      "Epoch: 760 cost: 18.5139306352\n",
      "Epoch: 761 cost: 18.5482029889\n",
      "Epoch: 762 cost: 18.5024287904\n",
      "Epoch: 763 cost: 18.5365853208\n",
      "Epoch: 764 cost: 18.4905192915\n",
      "Epoch: 765 cost: 18.5261796269\n",
      "Epoch: 766 cost: 18.4786686834\n",
      "Epoch: 767 cost: 18.5139451255\n",
      "Epoch: 768 cost: 18.4661296441\n",
      "Epoch: 769 cost: 18.502965137\n",
      "Epoch: 770 cost: 18.4535688999\n",
      "Epoch: 771 cost: 18.4906120706\n",
      "Epoch: 772 cost: 18.4403496004\n",
      "Epoch: 773 cost: 18.4779474786\n",
      "Epoch: 774 cost: 18.4265962433\n",
      "Epoch: 775 cost: 18.4640360259\n",
      "Epoch: 776 cost: 18.4121043467\n",
      "Epoch: 777 cost: 18.4485107495\n",
      "Epoch: 778 cost: 18.3967915824\n",
      "Epoch: 779 cost: 18.4312119142\n",
      "Epoch: 780 cost: 18.3806087882\n",
      "Epoch: 781 cost: 18.4121526797\n",
      "Epoch: 782 cost: 18.3634331036\n",
      "Epoch: 783 cost: 18.3919407751\n",
      "Epoch: 784 cost: 18.3458467126\n",
      "Epoch: 785 cost: 18.370332288\n",
      "Epoch: 786 cost: 18.3277748963\n",
      "Epoch: 787 cost: 18.3507576915\n",
      "Epoch: 788 cost: 18.3112246838\n",
      "Epoch: 789 cost: 18.3349764005\n",
      "Epoch: 790 cost: 18.2970343346\n",
      "Epoch: 791 cost: 18.3260121675\n",
      "Epoch: 792 cost: 18.2853997908\n",
      "Epoch: 793 cost: 18.3200814001\n",
      "Epoch: 794 cost: 18.2742031686\n",
      "Epoch: 795 cost: 18.3097370419\n",
      "Epoch: 796 cost: 18.26158232\n",
      "Epoch: 797 cost: 18.2955211832\n",
      "Epoch: 798 cost: 18.2479514962\n",
      "Epoch: 799 cost: 18.2777232223\n",
      "Epoch: 800 cost: 18.2338083513\n",
      "Epoch: 801 cost: 18.2592277933\n",
      "Epoch: 802 cost: 18.2193854185\n",
      "Epoch: 803 cost: 18.2388048083\n",
      "Epoch: 804 cost: 18.2044564169\n",
      "Epoch: 805 cost: 18.2165450124\n",
      "Epoch: 806 cost: 18.1887154478\n",
      "Epoch: 807 cost: 18.1911987891\n",
      "Epoch: 808 cost: 18.1712878803\n",
      "Epoch: 809 cost: 18.1630950773\n",
      "Epoch: 810 cost: 18.1500925713\n",
      "Epoch: 811 cost: 18.1357262566\n",
      "Epoch: 812 cost: 18.1253712532\n",
      "Epoch: 813 cost: 18.1179676855\n",
      "Epoch: 814 cost: 18.1141448224\n",
      "Epoch: 815 cost: 18.1125995086\n",
      "Epoch: 816 cost: 18.1110778552\n",
      "Epoch: 817 cost: 18.1106840176\n",
      "Epoch: 818 cost: 18.1125802328\n",
      "Epoch: 819 cost: 18.1175916049\n",
      "Epoch: 820 cost: 18.1266817907\n",
      "Epoch: 821 cost: 18.1416511701\n",
      "Epoch: 822 cost: 18.168234217\n",
      "Epoch: 823 cost: 18.2181802519\n",
      "Epoch: 824 cost: 18.2430410785\n",
      "Epoch: 825 cost: 18.2230162589\n",
      "Epoch: 826 cost: 18.1192761476\n",
      "Epoch: 827 cost: 18.0594123412\n",
      "Epoch: 828 cost: 18.0404747363\n",
      "Epoch: 829 cost: 18.0208692677\n",
      "Epoch: 830 cost: 17.9852345307\n",
      "Epoch: 831 cost: 17.9673865184\n",
      "Epoch: 832 cost: 17.9833224283\n",
      "Epoch: 833 cost: 18.0086805478\n",
      "Epoch: 834 cost: 18.00758209\n",
      "Epoch: 835 cost: 17.9924813585\n",
      "Epoch: 836 cost: 17.9624184611\n",
      "Epoch: 837 cost: 17.9390706452\n",
      "Epoch: 838 cost: 17.9372131\n",
      "Epoch: 839 cost: 17.9544319732\n",
      "Epoch: 840 cost: 17.9622519517\n",
      "Epoch: 841 cost: 17.955254329\n",
      "Epoch: 842 cost: 17.9385502174\n",
      "Epoch: 843 cost: 17.924112909\n",
      "Epoch: 844 cost: 17.9170360844\n",
      "Epoch: 845 cost: 17.9180058406\n",
      "Epoch: 846 cost: 17.9199268133\n",
      "Epoch: 847 cost: 17.9218667866\n",
      "Epoch: 848 cost: 17.9288388199\n",
      "Epoch: 849 cost: 17.9400630505\n",
      "Epoch: 850 cost: 17.9687257276\n",
      "Epoch: 851 cost: 18.0511595144\n",
      "Epoch: 852 cost: 18.1042903161\n",
      "Epoch: 853 cost: 18.0829149992\n",
      "Epoch: 854 cost: 18.0499954636\n",
      "Epoch: 855 cost: 17.8874476146\n",
      "Epoch: 856 cost: 17.8065856315\n",
      "Epoch: 857 cost: 17.7908460783\n",
      "Epoch: 858 cost: 17.8314624346\n",
      "Epoch: 859 cost: 17.8657877914\n",
      "Epoch: 860 cost: 17.8586957163\n",
      "Epoch: 861 cost: 17.8370261484\n",
      "Epoch: 862 cost: 17.8127306985\n",
      "Epoch: 863 cost: 17.8113147629\n",
      "Epoch: 864 cost: 17.8285649749\n",
      "Epoch: 865 cost: 17.8312290184\n",
      "Epoch: 866 cost: 17.8396654763\n",
      "Epoch: 867 cost: 17.8394947794\n",
      "Epoch: 868 cost: 17.8420759966\n",
      "Epoch: 869 cost: 17.8377086387\n",
      "Epoch: 870 cost: 17.8390687305\n",
      "Epoch: 871 cost: 17.823830758\n",
      "Epoch: 872 cost: 17.833429315\n",
      "Epoch: 873 cost: 17.8012924423\n",
      "Epoch: 874 cost: 17.8354671299\n",
      "Epoch: 875 cost: 17.7903628121\n",
      "Epoch: 876 cost: 17.8320499982\n",
      "Epoch: 877 cost: 17.7803673288\n",
      "Epoch: 878 cost: 17.8168696197\n",
      "Epoch: 879 cost: 17.7673725708\n",
      "Epoch: 880 cost: 17.7910172305\n",
      "Epoch: 881 cost: 17.7556539482\n",
      "Epoch: 882 cost: 17.7650419493\n",
      "Epoch: 883 cost: 17.766921895\n",
      "Epoch: 884 cost: 17.7522608054\n",
      "Epoch: 885 cost: 17.7561527681\n",
      "Epoch: 886 cost: 17.7563985675\n",
      "Epoch: 887 cost: 17.7457033637\n",
      "Epoch: 888 cost: 17.7463160024\n",
      "Epoch: 889 cost: 17.7413894103\n",
      "Epoch: 890 cost: 17.7364778734\n",
      "Epoch: 891 cost: 17.7333214987\n",
      "Epoch: 892 cost: 17.7288878604\n",
      "Epoch: 893 cost: 17.7239169282\n",
      "Epoch: 894 cost: 17.720617423\n",
      "Epoch: 895 cost: 17.7163632392\n",
      "Epoch: 896 cost: 17.7112131112\n",
      "Epoch: 897 cost: 17.7076018072\n",
      "Epoch: 898 cost: 17.7033278352\n",
      "Epoch: 899 cost: 17.6987624536\n",
      "Epoch: 900 cost: 17.694245386\n",
      "Epoch: 901 cost: 17.6904592533\n",
      "Epoch: 902 cost: 17.6862455549\n",
      "Epoch: 903 cost: 17.6815046865\n",
      "Epoch: 904 cost: 17.6773514272\n",
      "Epoch: 905 cost: 17.6732453837\n",
      "Epoch: 906 cost: 17.6687652976\n",
      "Epoch: 907 cost: 17.6642830442\n",
      "Epoch: 908 cost: 17.6602826328\n",
      "Epoch: 909 cost: 17.6558820865\n",
      "Epoch: 910 cost: 17.6514278562\n",
      "Epoch: 911 cost: 17.6474741945\n",
      "Epoch: 912 cost: 17.6429154074\n",
      "Epoch: 913 cost: 17.6383974216\n",
      "Epoch: 914 cost: 17.6346711168\n",
      "Epoch: 915 cost: 17.6300675901\n",
      "Epoch: 916 cost: 17.6250100358\n",
      "Epoch: 917 cost: 17.6215521953\n",
      "Epoch: 918 cost: 17.6180282457\n",
      "Epoch: 919 cost: 17.6118111446\n",
      "Epoch: 920 cost: 17.6076114945\n",
      "Epoch: 921 cost: 17.6064335869\n",
      "Epoch: 922 cost: 17.5993186698\n",
      "Epoch: 923 cost: 17.5922690687\n",
      "Epoch: 924 cost: 17.5940595529\n",
      "Epoch: 925 cost: 17.5906808028\n",
      "Epoch: 926 cost: 17.57478706\n",
      "Epoch: 927 cost: 17.5777290565\n",
      "Epoch: 928 cost: 17.5906423806\n",
      "Epoch: 929 cost: 17.5577090998\n",
      "Epoch: 930 cost: 17.5558957958\n",
      "Epoch: 931 cost: 17.5867543493\n",
      "Epoch: 932 cost: 17.559465291\n",
      "Epoch: 933 cost: 17.5291795287\n",
      "Epoch: 934 cost: 17.554504028\n",
      "Epoch: 935 cost: 17.596480095\n",
      "Epoch: 936 cost: 17.5114224445\n",
      "Epoch: 937 cost: 17.5351174613\n",
      "Epoch: 938 cost: 17.5769201403\n",
      "Epoch: 939 cost: 17.4943136972\n",
      "Epoch: 940 cost: 17.5147230448\n",
      "Epoch: 941 cost: 17.5631055794\n",
      "Epoch: 942 cost: 17.4798406134\n",
      "Epoch: 943 cost: 17.4986679757\n",
      "Epoch: 944 cost: 17.5417497678\n",
      "Epoch: 945 cost: 17.4695378326\n",
      "Epoch: 946 cost: 17.4817554228\n",
      "Epoch: 947 cost: 17.519826331\n",
      "Epoch: 948 cost: 17.471018549\n",
      "Epoch: 949 cost: 17.4621357119\n",
      "Epoch: 950 cost: 17.4888518212\n",
      "Epoch: 951 cost: 17.4904490747\n",
      "Epoch: 952 cost: 17.4420119207\n",
      "Epoch: 953 cost: 17.4640253516\n",
      "Epoch: 954 cost: 17.4859533976\n",
      "Epoch: 955 cost: 17.4324972617\n",
      "Epoch: 956 cost: 17.4563532998\n",
      "Epoch: 957 cost: 17.4473208582\n",
      "Epoch: 958 cost: 17.4327138485\n",
      "Epoch: 959 cost: 17.4397257833\n",
      "Epoch: 960 cost: 17.4296336891\n",
      "Epoch: 961 cost: 17.4315201951\n",
      "Epoch: 962 cost: 17.4198168833\n",
      "Epoch: 963 cost: 17.4256111098\n",
      "Epoch: 964 cost: 17.4068037823\n",
      "Epoch: 965 cost: 17.4261828116\n",
      "Epoch: 966 cost: 17.389554032\n",
      "Epoch: 967 cost: 17.4439656728\n",
      "Epoch: 968 cost: 17.3807351963\n",
      "Epoch: 969 cost: 17.4303845865\n",
      "Epoch: 970 cost: 17.368940704\n",
      "Epoch: 971 cost: 17.3975404869\n",
      "Epoch: 972 cost: 17.3547027377\n",
      "Epoch: 973 cost: 17.376675699\n",
      "Epoch: 974 cost: 17.3417733007\n",
      "Epoch: 975 cost: 17.3691251785\n",
      "Epoch: 976 cost: 17.3348336251\n",
      "Epoch: 977 cost: 17.3698742066\n",
      "Epoch: 978 cost: 17.3280310447\n",
      "Epoch: 979 cost: 17.3788750685\n",
      "Epoch: 980 cost: 17.3209643516\n",
      "Epoch: 981 cost: 17.3585879283\n",
      "Epoch: 982 cost: 17.3084358573\n",
      "Epoch: 983 cost: 17.3374099687\n",
      "Epoch: 984 cost: 17.2956408782\n",
      "Epoch: 985 cost: 17.3208895034\n",
      "Epoch: 986 cost: 17.2843443352\n",
      "Epoch: 987 cost: 17.3123419215\n",
      "Epoch: 988 cost: 17.2760407189\n",
      "Epoch: 989 cost: 17.3130440845\n",
      "Epoch: 990 cost: 17.2699446862\n",
      "Epoch: 991 cost: 17.3161543773\n",
      "Epoch: 992 cost: 17.2631293256\n",
      "Epoch: 993 cost: 17.3035991509\n",
      "Epoch: 994 cost: 17.2530307814\n",
      "Epoch: 995 cost: 17.2812161807\n",
      "Epoch: 996 cost: 17.2404278574\n",
      "Epoch: 997 cost: 17.2635609869\n",
      "Epoch: 998 cost: 17.2278752955\n",
      "Epoch: 999 cost: 17.2532223406\n",
      "Epoch: 1000 cost: 17.2184357745\n",
      "Optimization Finished!\n",
      "('Accuracy:', 0.64400923)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "Bsize = 256\n",
    "Tsize = len(x_train)\n",
    "Nbatch = int(Tsize/Bsize)\n",
    "if Tsize/(Bsize*1.0) != Nbatch:\n",
    "    Nbatch = Nbatch + 1\n",
    "print 'Total Number of Batches:',Nbatch\n",
    "nEpochs = 1000\n",
    "\n",
    "sess = tf.Session()\n",
    "# with tf.Session() as sess:\n",
    "sess.run(init)\n",
    "for epoch in range(nEpochs):\n",
    "    print \"Epoch:\",epoch+1,\n",
    "\n",
    "    avg_cost = 0.\n",
    "    for i in range(Nbatch):\n",
    "        if(i+1 == Nbatch):\n",
    "            a = (i)*Bsize+1\n",
    "            b = Tsize\n",
    "        else:\n",
    "            a = (i)*Bsize+1\n",
    "            b = (i+1)*Bsize\n",
    "\n",
    "        # print a,b\n",
    "        batch_x = x_train[a-1:b]\n",
    "        batch_y = lab_train[a-1:b]\n",
    "        # print np.shape(x_train),np.shape(batch_x)\n",
    "        _, c = sess.run([optimizer, cost], feed_dict={x: batch_x,\n",
    "                                                      y: batch_y})\n",
    "        avg_cost = avg_cost + c\n",
    "\n",
    "    print \"cost:\",avg_cost/(Nbatch*1.0)\n",
    "print(\"Optimization Finished!\")\n",
    "correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))\n",
    "\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))\n",
    "print(\"Accuracy:\", sess.run(accuracy, feed_dict = {x: x_test, y: lab_test}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Accuracy:', 0.77073735)\n"
     ]
    }
   ],
   "source": [
    "correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))\n",
    "\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))\n",
    "print(\"Accuracy:\", sess.run(accuracy, feed_dict = {x: x_test, y: lab_test}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "t1 = sess.run(correct_prediction,feed_dict = {x: x_test, y: lab_test})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "s1,s2 = sess.run([t3,t4] , feed_dict = {x: x_test, y: lab_test})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False, False,  True, ...,  True,  True,  True], dtype=bool)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1782.0\n"
     ]
    }
   ],
   "source": [
    "t2 = tf.reduce_sum(tf.cast(t1,\"float\"))\n",
    "print sess.run(t2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2604"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(t1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ -1.67639103e+01   1.43366706e+00  -1.73662201e+02]\n",
      " [ -2.30107754e-01   1.99578514e+01  -1.27069565e+02]\n",
      " [  5.67272043e+00  -5.49980011e+01  -2.42515545e+01]\n",
      " ..., \n",
      " [ -3.72981323e+03  -5.08399609e+03  -1.44776474e+02]\n",
      " [ -4.04178979e+03  -4.91788623e+03   8.46981873e+02]\n",
      " [ -3.63589966e+03  -5.14513574e+03  -2.30477264e+02]]\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "t1 = tf.argmax(pr,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "t2 = sess.run(t1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 1 0 ..., 2 2 2]\n"
     ]
    }
   ],
   "source": [
    "print t2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
